Add an archive-z repository mode
authorColin Walters <walters@verbum.org>
Sat, 15 Sep 2012 16:44:57 +0000 (12:44 -0400)
committerColin Walters <walters@verbum.org>
Sun, 23 Sep 2012 23:23:45 +0000 (19:23 -0400)
This is where loose content objects are stored as one compressed file,
instead of the two separate ones for regular archive mode.  This mode
would be suitable for HTTP servers, beause only one HTTP request is
necessary, and the result would be compressed.

18 files changed:
src/libostree/ostree-core.c
src/libostree/ostree-core.h
src/libostree/ostree-repo.c
src/libostree/ostree-repo.h
src/ostree/ostree-pull.c
src/ostree/ot-builtin-init.c
tests/archive-test.sh [new file with mode: 0755]
tests/libtest.sh
tests/t0000-basic.sh
tests/t0001-archive.sh
tests/t0002-archivez.sh [new file with mode: 0755]
tests/t0002-log.sh [deleted file]
tests/t0003-log.sh [new file with mode: 0755]
tests/t0003-remote-add.sh [deleted file]
tests/t0004-remote-add.sh [new file with mode: 0755]
tests/t0005-corruption.sh
tests/t0006-libarchive.sh
tests/t0010-pull.sh [deleted file]

index 03b8b4e4a0e2d73fac036a02c0030275f7c4e9a0..2906da186891f1af4d4162570caafad170549a70 100644 (file)
@@ -471,6 +471,34 @@ ostree_content_stream_parse (GInputStream           *input,
 
 }
 
+gboolean
+ostree_zlib_content_stream_open (GInputStream           *input,
+                                 guint64                *out_len,
+                                 GInputStream          **out_uncompressed,
+                                 GCancellable           *cancellable,
+                                 GError                **error)
+{
+  gboolean ret = FALSE;
+  guint64 uncompressed_len;
+  ot_lobj GConverter *zlib_decomp = NULL;
+  ot_lobj GInputStream *uncomp_input = NULL;
+
+  if (!g_input_stream_read_all (input, &uncompressed_len, sizeof (guint64),
+                                NULL, cancellable, error))
+    goto out;
+
+  uncompressed_len = GUINT64_FROM_BE (uncompressed_len);
+  zlib_decomp = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
+  uncomp_input = g_converter_input_stream_new (input, zlib_decomp);
+
+  if (out_len)
+    *out_len = uncompressed_len;
+  ot_transfer_out_value (out_uncompressed, &uncomp_input);
+  ret = TRUE;
+ out:
+  return ret;
+}
+
 gboolean
 ostree_content_file_parse (GFile                  *content_path,
                            gboolean                trusted,
@@ -909,8 +937,9 @@ ostree_checksum_bytes_peek (GVariant *bytes)
 }
 
 char *
-ostree_get_relative_object_path (const char *checksum,
-                                 OstreeObjectType type)
+ostree_get_relative_object_path (const char         *checksum,
+                                 OstreeObjectType    type,
+                                 gboolean            compressed)
 {
   GString *path;
 
@@ -923,6 +952,8 @@ ostree_get_relative_object_path (const char *checksum,
   g_string_append (path, checksum + 2);
   g_string_append_c (path, '.');
   g_string_append (path, ostree_object_type_to_string (type));
+  if (!OSTREE_OBJECT_TYPE_IS_META (type) && compressed)
+    g_string_append (path, "z");
 
   return g_string_free (path, FALSE);
 }
index 5b0381807ec98c3f630fe444ba285d1b01da9650..da0445076408937598a4f39d515237b10137813e 100644 (file)
@@ -127,7 +127,8 @@ void ostree_object_from_string (const char *str,
                                 OstreeObjectType *out_objtype);
 
 char *ostree_get_relative_object_path (const char        *checksum,
-                                       OstreeObjectType   type);
+                                       OstreeObjectType   type,
+                                       gboolean           compressed);
 
 char *ostree_get_relative_archive_content_path (const char        *checksum);
 
@@ -169,6 +170,12 @@ ostree_content_stream_parse (GInputStream           *input,
                              GCancellable           *cancellable,
                              GError                **error);
 
+gboolean ostree_zlib_content_stream_open (GInputStream           *input,
+                                          guint64                *out_len,
+                                          GInputStream          **out_uncompressed,
+                                          GCancellable           *cancellable,
+                                          GError                **error);
+
 gboolean ostree_content_file_parse (GFile                  *content_path,
                                     gboolean                trusted,
                                     GInputStream          **out_input,
index 30cb3e49608d1a1624929b5d13056d10cd6735cf..2d2f81defd8c3f9df36488584ce5a8270bf0bbd9 100644 (file)
@@ -582,6 +582,8 @@ ostree_repo_mode_from_string (const char      *mode,
     ret_mode = OSTREE_REPO_MODE_BARE;
   else if (strcmp (mode, "archive") == 0)
     ret_mode = OSTREE_REPO_MODE_ARCHIVE;
+  else if (strcmp (mode, "archive-z") == 0)
+    ret_mode = OSTREE_REPO_MODE_ARCHIVE_Z;
   else
     {
       g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -837,6 +839,7 @@ stage_object (OstreeRepo         *self,
   gboolean ret = FALSE;
   const char *actual_checksum;
   gboolean do_commit;
+  OstreeRepoMode repo_mode;
   ot_lobj GFileInfo *temp_info = NULL;
   ot_lobj GFile *temp_file = NULL;
   ot_lobj GFile *raw_temp_file = NULL;
@@ -863,6 +866,8 @@ stage_object (OstreeRepo         *self,
         goto out;
     }
 
+  repo_mode = ostree_repo_get_mode (self);
+
   if (out_csum)
     {
       checksum = g_checksum_new (G_CHECKSUM_SHA256);
@@ -870,7 +875,40 @@ stage_object (OstreeRepo         *self,
         checksum_input = ostree_checksum_input_stream_new (input, checksum);
     }
 
-  if (objtype == OSTREE_OBJECT_TYPE_FILE)
+  if (objtype == OSTREE_OBJECT_TYPE_FILE
+      && repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z)
+    {
+      gssize bytes_written;
+      guint64 len_be;
+      ot_lobj GOutputStream *raw_out_stream = NULL;
+      ot_lobj GConverter *zlib_compressor = NULL;
+      ot_lobj GOutputStream *compressed_out_stream = NULL;
+
+      if (!ostree_create_temp_regular_file (self->tmp_dir,
+                                            ostree_object_type_to_string (objtype), NULL,
+                                            &temp_file, &raw_out_stream,
+                                            cancellable, error))
+        goto out;
+
+      len_be = GUINT64_TO_BE (file_object_length);
+      if (!g_output_stream_write_all (raw_out_stream, (const guchar*) &len_be,
+                                      sizeof (guint64), NULL, cancellable, error))
+        goto out;
+      
+      zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9);
+      compressed_out_stream = g_converter_output_stream_new (raw_out_stream, zlib_compressor);
+      
+      bytes_written = g_output_stream_splice (compressed_out_stream,
+                                              checksum_input ? (GInputStream*)checksum_input : input,
+                                              G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
+                                              cancellable, error);
+      if (bytes_written < 0)
+        goto out;
+      
+      staged_raw_file = TRUE;
+      temp_file_is_regular = TRUE;
+    }
+  else if (objtype == OSTREE_OBJECT_TYPE_FILE)
     {
       ot_lobj GInputStream *file_input = NULL;
       ot_lobj GFileInfo *file_info = NULL;
@@ -884,7 +922,7 @@ stage_object (OstreeRepo         *self,
 
       temp_file_is_regular = g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR;
 
-      if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_BARE)
+      if (repo_mode == OSTREE_REPO_MODE_BARE)
         {
           if (!ostree_create_temp_file_from_input (self->tmp_dir,
                                                    ostree_object_type_to_string (objtype), NULL,
@@ -894,12 +932,12 @@ stage_object (OstreeRepo         *self,
             goto out;
           staged_raw_file = TRUE;
         }
-      else
+      else if (repo_mode == OSTREE_REPO_MODE_ARCHIVE)
         {
           ot_lvariant GVariant *file_meta = NULL;
           ot_lobj GInputStream *file_meta_input = NULL;
           ot_lobj GFileInfo *archive_content_file_info = NULL;
-
+          
           file_meta = ostree_file_header_new (file_info, xattrs);
           file_meta_input = ot_variant_read (file_meta);
 
@@ -947,6 +985,8 @@ stage_object (OstreeRepo         *self,
               staged_archive_file = TRUE;
             }
         }
+      else
+        g_assert_not_reached ();
     }
   else
     {
@@ -1127,6 +1167,7 @@ scan_loose_devino (OstreeRepo                     *self,
   gboolean ret = FALSE;
   GError *temp_error = NULL;
   guint i;
+  OstreeRepoMode repo_mode;
   ot_lptrarray GPtrArray *object_dirs = NULL;
   ot_lobj GFile *objdir = NULL;
 
@@ -1136,6 +1177,10 @@ scan_loose_devino (OstreeRepo                     *self,
         goto out;
     }
 
+  repo_mode = ostree_repo_get_mode (self);
+  if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z)
+    return TRUE;
+
   if (!get_loose_object_dirs (self, &object_dirs, cancellable, error))
     goto out;
 
@@ -1154,7 +1199,7 @@ scan_loose_devino (OstreeRepo                     *self,
         goto out;
 
       dirname = ot_gfile_get_basename_cached (objdir);
-  
+
       while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL)
         {
           const char *name;
@@ -1172,9 +1217,9 @@ scan_loose_devino (OstreeRepo                     *self,
               continue;
             }
       
-          if (!((ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE
+          if (!((repo_mode == OSTREE_REPO_MODE_ARCHIVE
                  && g_str_has_suffix (name, ".filecontent"))
-                || (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_BARE
+                || (repo_mode == OSTREE_REPO_MODE_BARE
                     && g_str_has_suffix (name, ".file"))))
             {
               g_clear_object (&file_info);
@@ -1358,14 +1403,17 @@ stage_directory_meta (OstreeRepo   *self,
 }
 
 GFile *
-ostree_repo_get_object_path (OstreeRepo  *self,
-                             const char    *checksum,
-                             OstreeObjectType type)
+ostree_repo_get_object_path (OstreeRepo       *self,
+                             const char       *checksum,
+                             OstreeObjectType  type)
 {
   char *relpath;
   GFile *ret;
+  gboolean compressed;
 
-  relpath = ostree_get_relative_object_path (checksum, type);
+  compressed = (type == OSTREE_OBJECT_TYPE_FILE
+                && ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE_Z);
+  relpath = ostree_get_relative_object_path (checksum, type, compressed);
   ret = g_file_resolve_relative_path (self->repodir, relpath);
   g_free (relpath);
  
@@ -1572,7 +1620,8 @@ ostree_repo_write_ref (OstreeRepo  *self,
   if (!write_checksum_file (dir, name, rev, error))
     goto out;
 
-  if (self->mode == OSTREE_REPO_MODE_ARCHIVE)
+  if (self->mode == OSTREE_REPO_MODE_ARCHIVE
+      || self->mode == OSTREE_REPO_MODE_ARCHIVE_Z)
     {
       if (!write_ref_summary (self, NULL, error))
         goto out;
@@ -2483,6 +2532,7 @@ ostree_repo_load_file (OstreeRepo         *self,
                        GError            **error)
 {
   gboolean ret = FALSE;
+  OstreeRepoMode repo_mode;
   ot_lvariant GVariant *file_data = NULL;
   ot_lobj GFile *loose_path = NULL;
   ot_lobj GFileInfo *content_loose_info = NULL;
@@ -2494,65 +2544,92 @@ ostree_repo_load_file (OstreeRepo         *self,
                          cancellable, error))
     goto out;
 
+  repo_mode = ostree_repo_get_mode (self);
+
   if (loose_path)
     {
-      if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE)
+      switch (repo_mode)
         {
-          ot_lvariant GVariant *archive_meta = NULL;
+        case OSTREE_REPO_MODE_ARCHIVE:
+          {
+            ot_lvariant GVariant *archive_meta = NULL;
 
-          if (!ot_util_variant_map (loose_path, OSTREE_FILE_HEADER_GVARIANT_FORMAT,
-                                    TRUE, &archive_meta, error))
-            goto out;
+            if (!ot_util_variant_map (loose_path, OSTREE_FILE_HEADER_GVARIANT_FORMAT,
+                                      TRUE, &archive_meta, error))
+              goto out;
 
-          if (!ostree_file_header_parse (archive_meta, &ret_file_info, &ret_xattrs,
-                                         error))
-            goto out;
+            if (!ostree_file_header_parse (archive_meta, &ret_file_info, &ret_xattrs,
+                                           error))
+              goto out;
 
-          if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR)
-            {
-              ot_lobj GFile *archive_content_path = NULL;
-              ot_lobj GFileInfo *content_info = NULL;
-
-              archive_content_path = ostree_repo_get_archive_content_path (self, checksum);
-              content_info = g_file_query_info (archive_content_path, OSTREE_GIO_FAST_QUERYINFO,
-                                                G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
-                                                cancellable, error);
-              if (!content_info)
-                goto out;
+            if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR)
+              {
+                ot_lobj GFile *archive_content_path = NULL;
+                ot_lobj GFileInfo *content_info = NULL;
 
-              if (out_input)
-                {
-                  ret_input = (GInputStream*)g_file_read (archive_content_path, cancellable, error);
-                  if (!ret_input)
-                    goto out;
-                }
-              g_file_info_set_size (ret_file_info, g_file_info_get_size (content_info));
-            }
-        }
-      else
-        {
-          ret_file_info = g_file_query_info (loose_path, OSTREE_GIO_FAST_QUERYINFO,
-                                             G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
-                                             cancellable, error);
-          if (!ret_file_info)
-            goto out;
+                archive_content_path = ostree_repo_get_archive_content_path (self, checksum);
+                content_info = g_file_query_info (archive_content_path, OSTREE_GIO_FAST_QUERYINFO,
+                                                  G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+                                                  cancellable, error);
+                if (!content_info)
+                  goto out;
 
-          if (out_xattrs)
-            {
-              if (!ostree_get_xattrs_for_file (loose_path, &ret_xattrs,
-                                               cancellable, error))
-                goto out;
-            }
+                if (out_input)
+                  {
+                    ret_input = (GInputStream*)g_file_read (archive_content_path, cancellable, error);
+                    if (!ret_input)
+                      goto out;
+                  }
+                g_file_info_set_size (ret_file_info, g_file_info_get_size (content_info));
+              }
+          }
+          break;
+        case OSTREE_REPO_MODE_ARCHIVE_Z:
+          {
+            ot_lobj GInputStream *file_in = NULL;
+            ot_lobj GInputStream *uncomp_input = NULL;
+            guint64 uncompressed_len;
+
+            file_in = (GInputStream*)g_file_read (loose_path, cancellable, error);
+            if (!file_in)
+              goto out;
 
-          if (out_input && g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR)
-            {
-              ret_input = (GInputStream*) g_file_read (loose_path, cancellable, error);
-              if (!ret_input)
-                {
-                  g_prefix_error (error, "Error opening loose file object %s: ", ot_gfile_get_path_cached (loose_path));
+            if (!ostree_zlib_content_stream_open (file_in, &uncompressed_len, &uncomp_input, 
+                                                  cancellable, error))
+              goto out;
+
+            if (!ostree_content_stream_parse (uncomp_input, uncompressed_len, TRUE,
+                                              &ret_input, &ret_file_info, &ret_xattrs,
+                                              cancellable, error))
+              goto out;
+          }
+          break;
+        case OSTREE_REPO_MODE_BARE:
+          {
+            ret_file_info = g_file_query_info (loose_path, OSTREE_GIO_FAST_QUERYINFO,
+                                               G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+                                               cancellable, error);
+            if (!ret_file_info)
+              goto out;
+
+            if (out_xattrs)
+              {
+                if (!ostree_get_xattrs_for_file (loose_path, &ret_xattrs,
+                                                 cancellable, error))
                   goto out;
-                }
-            }
+              }
+
+            if (out_input && g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR)
+              {
+                ret_input = (GInputStream*) g_file_read (loose_path, cancellable, error);
+                if (!ret_input)
+                  {
+                    g_prefix_error (error, "Error opening loose file object %s: ", ot_gfile_get_path_cached (loose_path));
+                    goto out;
+                  }
+              }
+          }
+          break;
         }
     }
   else if (self->parent_repo)
index 2f31bef0ae3b8db88931d59dfe1c123869a65051..3e57d2481cd014168ddf04ba77e74f7c8e44f2fe 100644 (file)
@@ -44,7 +44,8 @@ GFile *       ostree_repo_get_path (OstreeRepo  *self);
 
 typedef enum {
   OSTREE_REPO_MODE_BARE,
-  OSTREE_REPO_MODE_ARCHIVE
+  OSTREE_REPO_MODE_ARCHIVE,
+  OSTREE_REPO_MODE_ARCHIVE_Z
 } OstreeRepoMode;
 
 gboolean       ostree_repo_mode_from_string (const char      *mode,
index b00c9d27376def186af35bcb7fa885a166cb7fbb..3ccd598d3e105ee419cc05eb80d590e535c10c36 100644 (file)
@@ -80,6 +80,7 @@ static GOptionEntry options[] = {
 typedef struct {
   OstreeRepo   *repo;
   char         *remote_name;
+  OstreeRepoMode remote_mode;
   OstreeFetcher *fetcher;
   SoupURI      *base_uri;
 
@@ -332,7 +333,8 @@ fetch_loose_object (OtPullData  *pull_data,
   ot_lobj GFile *ret_temp_path = NULL;
   SoupURI *obj_uri = NULL;
 
-  objpath = ostree_get_relative_object_path (checksum, objtype);
+  objpath = ostree_get_relative_object_path (checksum, objtype,
+                                             pull_data->remote_mode == OSTREE_REPO_MODE_ARCHIVE_Z);
   obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
   
   if (!fetch_uri (pull_data, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path,
@@ -639,6 +641,7 @@ content_fetch_on_checksum_complete (GObject        *object,
   GError **error = &local_error;
   guint64 length;
   GCancellable *cancellable = NULL;
+  gboolean compressed;
   ot_lfree guchar *csum;
   ot_lvariant GVariant *file_meta = NULL;
   ot_lobj GFileInfo *file_info = NULL;
@@ -651,25 +654,6 @@ content_fetch_on_checksum_complete (GObject        *object,
   if (!csum)
     goto out;
 
-  if (!ot_util_variant_map (data->meta_path, OSTREE_FILE_HEADER_GVARIANT_FORMAT, FALSE,
-                            &file_meta, error))
-    goto out;
-  
-  if (!ostree_file_header_parse (file_meta, &file_info, &xattrs, error))
-    goto out;
-
-  if (data->content_path)
-    {
-      content_input = (GInputStream*)g_file_read (data->content_path, cancellable, error);
-      if (!content_input)
-        goto out;
-    }
-
-  if (!ostree_raw_file_to_content_stream (content_input, file_info, xattrs,
-                                          &file_object_input, &length,
-                                          cancellable, error))
-    goto out;
-
   checksum = ostree_checksum_from_bytes (csum);
 
   if (strcmp (checksum, data->checksum) != 0)
@@ -680,6 +664,39 @@ content_fetch_on_checksum_complete (GObject        *object,
       goto out;
     }
 
+  compressed = data->pull_data->remote_mode == OSTREE_REPO_MODE_ARCHIVE_Z;
+
+  if (compressed)
+    {
+      content_input = (GInputStream*)g_file_read (data->content_path, cancellable, error);
+      if (!content_input)
+        goto out;
+      if (!ostree_zlib_content_stream_open (content_input, &length, &file_object_input, 
+                                            cancellable, error))
+        goto out;
+    }
+  else
+    {
+      if (!ot_util_variant_map (data->meta_path, OSTREE_FILE_HEADER_GVARIANT_FORMAT, TRUE,
+                                &file_meta, error))
+        goto out;
+  
+      if (!ostree_file_header_parse (file_meta, &file_info, &xattrs, error))
+        goto out;
+
+      if (data->content_path)
+        {
+          content_input = (GInputStream*)g_file_read (data->content_path, cancellable, error);
+          if (!content_input)
+            goto out;
+        }
+      
+      if (!ostree_raw_file_to_content_stream (content_input, file_info, xattrs,
+                                              &file_object_input, &length,
+                                              cancellable, error))
+        goto out;
+    }
+
   if (!ostree_repo_stage_content_trusted (data->pull_data->repo, checksum,
                                           file_object_input, length,
                                           cancellable, error))
@@ -702,6 +719,7 @@ content_fetch_on_complete (GObject        *object,
   OtFetchOneContentItemData *data = user_data;
   GError *local_error = NULL;
   GError **error = &local_error;
+  gboolean compressed;
   GCancellable *cancellable = NULL;
   gboolean was_content_fetch = FALSE;
   gboolean need_content_fetch = FALSE;
@@ -711,6 +729,7 @@ content_fetch_on_complete (GObject        *object,
   ot_lobj GInputStream *file_object_input = NULL;
   ot_lvariant GVariant *xattrs = NULL;
 
+  compressed = data->pull_data->remote_mode == OSTREE_REPO_MODE_ARCHIVE_Z;
   was_content_fetch = data->fetching_content;
 
   if (was_content_fetch)
@@ -752,7 +771,25 @@ content_fetch_on_complete (GObject        *object,
         }
     }
 
-  if (!need_content_fetch)
+  if (!need_content_fetch && compressed)
+    {
+      ot_lobj GInputStream *uncomp_input = NULL;
+      guint64 uncompressed_len;
+
+      g_assert (data->content_path != NULL);
+      content_input = (GInputStream*)g_file_read (data->content_path, cancellable, error);
+      if (!content_input)
+        goto out;
+
+      if (!ostree_zlib_content_stream_open (content_input, &uncompressed_len, &uncomp_input, 
+                                            cancellable, error))
+        goto out;
+      
+      data->pull_data->outstanding_checksum_requests++;
+      ot_gio_checksum_stream_async (uncomp_input, G_PRIORITY_DEFAULT, NULL,
+                                    content_fetch_on_checksum_complete, data);
+    }
+  else if (!need_content_fetch)
     {
       if (data->content_path)
         {
@@ -806,20 +843,24 @@ enqueue_loose_meta_requests (OtPullData *pull_data)
       ot_lfree char *objpath = NULL;
       SoupURI *obj_uri = NULL;
       OtFetchOneContentItemData *one_item_data;
+      gboolean compressed = pull_data->remote_mode == OSTREE_REPO_MODE_ARCHIVE_Z;
           
       one_item_data = g_new0 (OtFetchOneContentItemData, 1);
       one_item_data->pull_data = pull_data;
       one_item_data->checksum = g_strdup (checksum);
-      one_item_data->fetching_content = FALSE;
+      one_item_data->fetching_content = compressed;
           
-      objpath = ostree_get_relative_object_path (checksum, OSTREE_OBJECT_TYPE_FILE);
+      objpath = ostree_get_relative_object_path (checksum, OSTREE_OBJECT_TYPE_FILE, compressed);
       obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
 
       ostree_fetcher_request_uri_async (pull_data->fetcher, obj_uri, cancellable,
                                         content_fetch_on_complete, one_item_data);
       soup_uri_free (obj_uri);
 
-      pull_data->outstanding_filemeta_requests++;
+      if (compressed)
+        pull_data->outstanding_filecontent_requests++;
+      else
+        pull_data->outstanding_filemeta_requests++;
       g_hash_table_iter_remove (&hash_iter);
 
       /* Don't let too many requests queue up; when we're fetching
@@ -1026,9 +1067,9 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
   gpointer key, value;
   int i;
   GCancellable *cancellable = NULL;
-  OstreeRepoMode remote_repo_mode;
   ot_lfree char *remote_key = NULL;
   ot_lobj OstreeRepo *repo = NULL;
+  ot_lfree char *remote_config_content = NULL;
   ot_lfree char *path = NULL;
   ot_lfree char *baseurl = NULL;
   ot_lfree char *summary_data = NULL;
@@ -1094,12 +1135,13 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
                                           &remote_mode_str, error))
     goto out;
 
-  if (!ostree_repo_mode_from_string (remote_mode_str, &remote_repo_mode, error))
+  if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
     goto out;
 
-  switch (remote_repo_mode)
+  switch (pull_data->remote_mode)
     {
     case OSTREE_REPO_MODE_ARCHIVE:
+    case OSTREE_REPO_MODE_ARCHIVE_Z:
       break;
     default:
       g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
index 0707242e7f8149c4208c3a519d9fbb11554cabd2..8dd282174942c5e5d7554f8c4b90d8eb15d185cb 100644 (file)
 
 #include <glib/gi18n.h>
 
-static gboolean archive;
+static gboolean opt_archive;
+static char *opt_mode = NULL;
 
 static GOptionEntry options[] = {
-  { "archive", 0, 0, G_OPTION_ARG_NONE, &archive, "Initialize repository as archive", NULL },
+  { "archive", 0, 0, G_OPTION_ARG_NONE, &opt_archive, "Initialize repository as archive", NULL },
+  { "mode", 0, 0, G_OPTION_ARG_STRING, &opt_mode, "Initialize repository in given mode (bare, archive, archive-z)", NULL },
   { NULL }
 };
 
@@ -44,6 +46,7 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
   GOptionContext *context = NULL;
   gboolean ret = FALSE;
   __attribute__ ((unused)) GCancellable *cancellable = NULL;
+  const char *mode_str = "bare";
   ot_lobj GFile *child = NULL;
   ot_lobj GFile *grandchild = NULL;
   ot_lobj OstreeRepo *repo = NULL;
@@ -58,7 +61,16 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
   child = g_file_get_child (repo_path, "config");
 
   config_data = g_string_new (DEFAULT_CONFIG_CONTENTS);
-  g_string_append_printf (config_data, "mode=%s\n", archive ? "archive" : "bare");
+  if (opt_archive)
+    mode_str = "archive";
+  else if (opt_mode)
+    {
+      OstreeRepoMode mode;
+      if (!ostree_repo_mode_from_string (opt_mode, &mode, error))
+        goto out;
+      mode_str = opt_mode;
+    }
+  g_string_append_printf (config_data, "mode=%s\n", mode_str);
   if (!g_file_replace_contents (child,
                                 config_data->str,
                                 config_data->len,
diff --git a/tests/archive-test.sh b/tests/archive-test.sh
new file mode 100755 (executable)
index 0000000..cd22b87
--- /dev/null
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+$OSTREE checkout test2 checkout-test2
+echo "ok checkout"
+
+cd checkout-test2
+assert_has_file firstfile
+assert_has_file baz/cow
+assert_file_has_content baz/cow moo
+assert_has_file baz/deeper/ohyeah
+echo "ok content"
+
+cd ${test_tmpdir}
+mkdir repo2
+${CMD_PREFIX} ostree --repo=repo2 init
+${CMD_PREFIX} ostree --repo=repo2 pull-local repo
+echo "ok local clone"
+
+cd ${test_tmpdir}
+${CMD_PREFIX} ostree --repo=repo2 checkout test2 test2-checkout-from-local-clone
+cd test2-checkout-from-local-clone
+assert_file_has_content baz/cow moo
+echo "ok local clone checkout"
+
+$OSTREE checkout -U test2 checkout-user-test2
+echo "ok user checkout"
+
+cd ${test_tmpdir}/checkout-test2
+$OSTREE commit -b test2-uid0 -s 'UID 0 test' --owner-uid=0 --owner-gid=0
+echo "ok uid0 commit"
+
+cd ${test_tmpdir}
+$OSTREE ls test2-uid0 /firstfile > uid0-ls-output.txt
+assert_file_has_content uid0-ls-output.txt "-00664 0 0      6 /firstfile" 
+echo "ok uid0 ls"
+
+$OSTREE checkout -U test2-uid0 checkout-user-test2-uid0
+echo "ok user checkout from uid 0"
+
+cd ${test_tmpdir}
+$OSTREE cat test2 /baz/cow > cow-contents
+assert_file_has_content cow-contents "moo"
+echo "ok cat-file"
+
+cd ${test_tmpdir}
+$OSTREE fsck
+echo "ok fsck"
index c09639f487d2a273bc878f4eaa4d6ec89c6c9308..93f8a7d4d2a13ce1274e0ba89c61b3360ff43e8c 100644 (file)
@@ -75,8 +75,8 @@ setup_test_repository () {
     cd repo
     ot_repo="--repo=`pwd`"
     export OSTREE="${CMD_PREFIX} ostree ${ot_repo}"
-    if test "$mode" = "archive"; then
-       $OSTREE init --archive
+    if test -n "$mode"; then
+       $OSTREE init --mode=${mode}
     else
        $OSTREE init
     fi
@@ -110,11 +110,13 @@ setup_test_repository () {
 }
 
 setup_fake_remote_repo1() {
+    mode=$1
+    shift
     oldpwd=`pwd`
     mkdir ostree-srv
     cd ostree-srv
     mkdir gnomerepo
-    ${CMD_PREFIX} ostree --repo=gnomerepo init --archive
+    ${CMD_PREFIX} ostree --repo=gnomerepo init --mode=$mode
     mkdir gnomerepo-files
     cd gnomerepo-files 
     echo first > firstfile
index dbf674116adbee5322347698d5d9e57366e632b5..a72d31dcb683e617156e0f23f247e50d94d17ebf 100755 (executable)
@@ -23,7 +23,7 @@ echo "1..28"
 
 . libtest.sh
 
-setup_test_repository "regular"
+setup_test_repository "bare"
 echo "ok setup"
 
 $OSTREE checkout test2 checkout-test2
index 0e9f88cfd937089bf2b18974cf3c1e961ef508bf..642cdf9ed17c1c148a3ab04005fc96539651b2c6 100755 (executable)
@@ -26,48 +26,4 @@ echo '1..11'
 setup_test_repository "archive"
 echo "ok setup"
 
-$OSTREE checkout test2 checkout-test2
-echo "ok checkout"
-
-cd checkout-test2
-assert_has_file firstfile
-assert_has_file baz/cow
-assert_file_has_content baz/cow moo
-assert_has_file baz/deeper/ohyeah
-echo "ok content"
-
-cd ${test_tmpdir}
-mkdir repo2
-${CMD_PREFIX} ostree --repo=repo2 init
-${CMD_PREFIX} ostree --repo=repo2 pull-local repo
-echo "ok local clone"
-
-cd ${test_tmpdir}
-${CMD_PREFIX} ostree --repo=repo2 checkout test2 test2-checkout-from-local-clone
-cd test2-checkout-from-local-clone
-assert_file_has_content baz/cow moo
-echo "ok local clone checkout"
-
-$OSTREE checkout -U test2 checkout-user-test2
-echo "ok user checkout"
-
-cd ${test_tmpdir}/checkout-test2
-$OSTREE commit -b test2-uid0 -s 'UID 0 test' --owner-uid=0 --owner-gid=0
-echo "ok uid0 commit"
-
-cd ${test_tmpdir}
-$OSTREE ls test2-uid0 /firstfile > uid0-ls-output.txt
-assert_file_has_content uid0-ls-output.txt "-00664 0 0      6 /firstfile" 
-echo "ok uid0 ls"
-
-$OSTREE checkout -U test2-uid0 checkout-user-test2-uid0
-echo "ok user checkout from uid 0"
-
-cd ${test_tmpdir}
-$OSTREE cat test2 /baz/cow > cow-contents
-assert_file_has_content cow-contents "moo"
-echo "ok cat-file"
-
-cd ${test_tmpdir}
-$OSTREE fsck
-echo "ok fsck"
+. ${SRCDIR}/archive-test.sh
diff --git a/tests/t0002-archivez.sh b/tests/t0002-archivez.sh
new file mode 100755 (executable)
index 0000000..46248ea
--- /dev/null
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. libtest.sh
+
+echo '1..11'
+
+setup_test_repository "archive-z"
+echo "ok setup"
+
+. ${SRCDIR}/archive-test.sh
diff --git a/tests/t0002-log.sh b/tests/t0002-log.sh
deleted file mode 100755 (executable)
index dddd7db..0000000
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-#
-# Copyright (C) 2011 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-set -e
-
-. libtest.sh
-
-echo "1..1"
-
-setup_test_repository "regular"
-$OSTREE log test2 > $test_tmpdir/log.txt
-assert_file_has_content $test_tmpdir/log.txt "Test Commit 1"
-assert_file_has_content $test_tmpdir/log.txt "Test Commit 2"
-echo "ok log"
diff --git a/tests/t0003-log.sh b/tests/t0003-log.sh
new file mode 100755 (executable)
index 0000000..2fb5ea8
--- /dev/null
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. libtest.sh
+
+echo "1..1"
+
+setup_test_repository "bare"
+$OSTREE log test2 > $test_tmpdir/log.txt
+assert_file_has_content $test_tmpdir/log.txt "Test Commit 1"
+assert_file_has_content $test_tmpdir/log.txt "Test Commit 2"
+echo "ok log"
diff --git a/tests/t0003-remote-add.sh b/tests/t0003-remote-add.sh
deleted file mode 100755 (executable)
index 6310b2d..0000000
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-#
-# Copyright (C) 2011 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-set -e
-
-. libtest.sh
-
-echo '1..2'
-
-setup_test_repository "regular"
-$OSTREE remote add origin http://example.com/ostree/gnome
-echo "ok remote add"
-assert_file_has_content $test_tmpdir/repo/config "example.com"
-echo "ok config"
diff --git a/tests/t0004-remote-add.sh b/tests/t0004-remote-add.sh
new file mode 100755 (executable)
index 0000000..ffab0fa
--- /dev/null
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. libtest.sh
+
+echo '1..2'
+
+setup_test_repository "bare"
+$OSTREE remote add origin http://example.com/ostree/gnome
+echo "ok remote add"
+assert_file_has_content $test_tmpdir/repo/config "example.com"
+echo "ok config"
index 5d4bb60fabe30c3191ad7c58f168c37a739b72e8..3961d46b87ebcdaadd469c5700d4f7500d9bbffb 100755 (executable)
@@ -23,7 +23,7 @@ echo "1..1"
 
 . libtest.sh
 
-setup_test_repository "regular"
+setup_test_repository "bare"
 $OSTREE checkout test2 checkout-test2
 cd checkout-test2
 chmod o+x firstfile
index 718109a2a6c2b8a241d681dac5c4c58c43c4bb9c..d941f472078f4c58a0a5b9875267537963b4b83c 100755 (executable)
@@ -23,7 +23,7 @@ echo "1..7"
 
 . libtest.sh
 
-setup_test_repository "regular"
+setup_test_repository "bare"
 cd ${test_tmpdir}
 mkdir foo
 cd foo
diff --git a/tests/t0010-pull.sh b/tests/t0010-pull.sh
deleted file mode 100755 (executable)
index 1fe002e..0000000
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-#
-# Copyright (C) 2011 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-set -e
-
-. libtest.sh
-
-echo '1..2'
-
-setup_fake_remote_repo1
-cd ${test_tmpdir}
-mkdir repo
-${CMD_PREFIX} ostree --repo=repo init
-${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo
-${CMD_PREFIX} ostree-pull --repo=repo origin main
-${CMD_PREFIX} ostree --repo=repo fsck
-echo "ok pull"
-
-cd ${test_tmpdir}
-$OSTREE checkout origin/main checkout-origin-main
-cd checkout-origin-main
-assert_file_has_content firstfile '^first$'
-assert_file_has_content baz/cow '^moo$'
-echo "ok pull contents"